{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, loss 0.0038, train acc 0.750, test acc 0.945\n",
      "epoch 2, loss 0.0010, train acc 0.941, test acc 0.961\n",
      "epoch 3, loss 0.0007, train acc 0.958, test acc 0.966\n",
      "epoch 4, loss 0.0006, train acc 0.967, test acc 0.970\n",
      "epoch 5, loss 0.0005, train acc 0.970, test acc 0.975\n",
      "epoch 6, loss 0.0004, train acc 0.973, test acc 0.974\n",
      "epoch 7, loss 0.0004, train acc 0.976, test acc 0.975\n",
      "epoch 8, loss 0.0004, train acc 0.977, test acc 0.978\n",
      "epoch 9, loss 0.0003, train acc 0.980, test acc 0.978\n",
      "epoch 10, loss 0.0003, train acc 0.981, test acc 0.978\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEGCAYAAABCa2PoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXRc5Z3n//dXpX21LS94kbEAs5jFcjBmC5CETjCEiaEbgpNJGrrJMMwPpqHTZGL3JJmEc5hDJmkgnBAICRBCmBhCIPF0SCABEkID3ojACzgWtsHybtmSJdlaSvr+/rhXUqlcWixXqUrS53VOHd2697lPPbds6+PnPvc+19wdERGRY5WV7gaIiMjooEAREZGkUKCIiEhSKFBERCQpFCgiIpIU2eluQDpNnDjRZ82ale5miIiMKGvWrNnn7pPi14/pQJk1axarV69OdzNEREYUM/sg0Xqd8hIRkaRQoIiISFIoUEREJCnG9BiKiIw+7e3t1NbW0tLSku6mjHj5+fnMmDGDnJycQZVXoIjIqFJbW0tJSQmzZs3CzNLdnBHL3amrq6O2tpbKyspB7aNTXiIyqrS0tFBeXq4wOUZmRnl5+VH19BQoR6mz09m8t4ln36pl3faGdDdHRBJQmCTH0X6POuV1lO57aRP3v7QJgJsuPoEzppeluUUiIplBPZSjdPq00u7l6g/r09gSEZHMokA5SvMqxnUvr93eQLSjM42tEZFMU19fzw9+8IOj3u+KK66gvv7o/5N6ww038Mwzzxz1fqmgQDlKk0vzmVqWD8Dh9g7+urspzS0SkUzSV6B0dHT0u9/zzz/PuHHj+i2T6TSGMgRVFePY2bALgLdr65kTcxpMRDLHrCW/SVndW+/+dML1S5Ys4f3336eqqoqcnByKi4uZOnUq1dXVbNiwgauuuopt27bR0tLCbbfdxk033RS0NZxbsKmpicsvv5yPfvSjvP7660yfPp1f//rXFBQUDNiml156iTvuuINoNMo555zDgw8+SF5eHkuWLGH58uVkZ2fzqU99iu9+97v84he/4Fvf+haRSISysjJeffXVY/5O1EMZgrkxp700jiIise6++25OPPFEqqur+c53vsPKlSu566672LBhAwCPPvooa9asYfXq1dx///3U1dUdUcemTZu45ZZbWL9+PePGjeOXv/zlgJ/b0tLCDTfcwFNPPcXatWuJRqM8+OCD7N+/n+eee47169fzzjvv8LWvfQ2AO++8kxdeeIG3336b5cuXJ+XYFShDUBUTKG/XKlBEpG8LFizodWPg/fffz9y5cznvvPPYtm0bmzZtOmKfyspKqqqqADj77LPZunXrgJ+zceNGKisrOfnkkwG4/vrrefXVVyktLSU/P58vfelLPPvssxQWFgJw4YUXcsMNN/CjH/1owNNxg6VTXkNw5vQysgw6Hf66u5Hm1ihFefoqRTJNX6elhlNRUVH38h//+Ef+8Ic/8MYbb1BYWMjHPvaxhDcO5uXldS9HIhEOHz484Oe4e8L12dnZrFy5kpdeeolly5bx/e9/n5dffpmHHnqIFStW8Jvf/Iaqqiqqq6spLy8fwhH2UA9lCIrysjl5SgkQhMpa3eAoIqGSkhIaGxsTbmtoaGD8+PEUFhby3nvv8eabbybtc0899VS2bt1KTU0NAE888QSXXHIJTU1NNDQ0cMUVV3DfffdRXV0NwPvvv8+5557LnXfeycSJE9m2bdsxt0H/rR6iuTPG8d6u4C9N9bZ6zjvh2JJdREaH8vJyLrzwQs444wwKCgqYMmVK97aFCxfy0EMPcdZZZ3HKKadw3nnnJe1z8/Pzeeyxx7j22mu7B+Vvvvlm9u/fz6JFi2hpacHduffeewH4yle+wqZNm3B3Lr30UubOnXvMbbC+ukljwfz5832oT2z8+coPWfrsWgAuP+M4HvzC2clsmogM0bvvvstpp52W7maMGom+TzNb4+7z48vqlNcQzZ0Rc6XXNg3Mi4jolNcQnTylmIKcCIfbO9jZ0MLugy1MKc1Pd7NEZJS65ZZb+I//+I9e62677Tb+4R/+IU0tOpICZYiyI1mcOb2MlVv3A0Ev5bLTj0tzq0RktHrggQfS3YQBpfSUl5ktNLONZlZjZksSbM8zs6fC7SvMbFbMtqXh+o1mdlm4Lt/MVprZ22a23sy+FVP+J2a2xcyqw1dVKo8NoGpmzP0oOu0lImNcynooZhYBHgA+CdQCq8xsubtviCl2I3DA3U8ys8XAt4HrzGwOsBg4HZgG/MHMTgZagU+4e5OZ5QCvmdlv3b3r2ruvuPuwzZKmcRQRkR6p7KEsAGrcfbO7twHLgEVxZRYBj4fLzwCXWvBEl0XAMndvdfctQA2wwANdszHmhK+0XaYW20N5p7aBzs6xe8WciEgqA2U6EHunTG24LmEZd48CDUB5f/uaWcTMqoE9wO/dfUVMubvM7B0zu9fM8kixaWX5TCwOPqapNcr7ezXzsIiMXakMlETPjoz/L3xfZfrc19073L0KmAEsMLMzwu1LgVOBc4AJwFcTNsrsJjNbbWar9+7dO/BR9MPMes3rpdNeIjLU56EA3HfffRw6dKjfMrNmzWLfvn1Dqj/VUhkotUBFzPsZwI6+yphZNlAG7B/Mvu5eD/wRWBi+3xmeEmsFHiM45XYEd3/Y3ee7+/xJkyYN7chiVFX0PAJYgSIiqQ6UTJbKy4ZXAbPNrBLYTjDI/vm4MsuB64E3gGuAl93dzWw58H/N7B6CQfnZwEozmwS0u3u9mRUAf0MwkI+ZTXX3neEYzFXAuhQeW7eqivHdy5p5WCTD/HYJ7Fqb3DqPOxMuv7vPzbHPQ/nkJz/J5MmTefrpp2ltbeXqq6/mW9/6Fs3NzXz2s5+ltraWjo4Ovv71r7N792527NjBxz/+cSZOnMgrr7wyYFPuueceHn30UQC+9KUvcfvttyes+7rrrkv4TJRkS1mguHvUzG4FXgAiwKPuvt7M7gRWu/ty4BHgCTOrIeiZLA73XW9mTwMbgChwi7t3mNlU4PHwCrIs4Gl3//fwI58MA8eAauDmVB1brDNn9PRQ3tvZSEt7B/k5keH4aBHJQHfffTfr1q2jurqaF198kWeeeYaVK1fi7nzmM5/h1VdfZe/evUybNo3f/CZ4AFhDQwNlZWXcc889vPLKK0ycOHHAz1mzZg2PPfYYK1aswN0599xzueSSS9i8efMRdXc9E+W9997DzIb0qOHBSOmNje7+PPB83LpvxCy3ANf2se9dwF1x694B5vVR/hPH2t6hKCvI4cRJRby/t5lop7N+RwNnHz8hHU0RkXj99CSGw4svvsiLL77IvHnBr62mpiY2bdrERRddxB133MFXv/pVrrzySi666KKjrvu1117j6quv7p4e/2//9m/585//zMKFC4+oOxqNdj8T5dOf/jRXXnllUo+zi+bySoLYJzj+RU9wFJGQu7N06VKqq6uprq6mpqaGG2+8kZNPPpk1a9Zw5plnsnTpUu68884h1Z1Iorq7nonyd3/3d/zqV79i4cKFx3poCSlQkmBeryc46tkoImNZ7PNQLrvsMh599FGamoJbCrZv386ePXvYsWMHhYWFfOELX+COO+7grbfeOmLfgVx88cX86le/4tChQzQ3N/Pcc89x0UUXJay7r2eiJJvm8kqCXs+Y33YgjS0RkXSLfR7K5Zdfzuc//3nOP/98AIqLi/nZz35GTU0NX/nKV8jKyiInJ4cHH3wQgJtuuonLL7+cqVOnDjgo/5GPfIQbbriBBQuCC1q/9KUvMW/ePF544YUj6m5sbEz4TJRk0/NQhvg8lFht0U7O+OYLtEU7AVjztb+hvDjl91WKSAJ6Hkpy6Xkowyw3O4vTp5V2v9flwyIyFumUV5JUVYzrHpCv3tbAJ06dMsAeIiJ9O/fcc2ltbe217oknnuDMM89MU4sGpkBJEk3BIpI53J3gHueRa8WKFQMXSrGjHRLRKa8kiQ2Ut7fVH/UfhIgkR35+PnV1dfo3eIzcnbq6OvLzB/8kWvVQkmTmhELGF+Zw4FA7DYfb2Vp3iMqJRelulsiYM2PGDGpraznWyV8lCOcZM2YMurwCJUnMjLkV4/jjxuAv8dvb6hUoImmQk5NDZWVlupsxJumUVxLpCY4iMpYpUJIo9gmOChQRGWsUKEkU20PZsOMgrdGONLZGRGR4KVCSaEJRLseXFwLQ1tHJezsHNyePiMhooEBJMo2jiMhYpUBJsvj7UURExgoFSpLN1R3zIjJGKVCS7PRppWRnBVM+bN7XTMOh9jS3SERkeChQkiw/J8JpUzXzsIiMPQqUFNA4ioiMRSkNFDNbaGYbzazGzJYk2J5nZk+F21eY2ayYbUvD9RvN7LJwXb6ZrTSzt81svZl9K6Z8ZVjHprDO3FQeW380jiIiY1HKAsXMIsADwOXAHOBzZjYnrtiNwAF3Pwm4F/h2uO8cYDFwOrAQ+EFYXyvwCXefC1QBC83svLCubwP3uvts4EBYd1r06qHUauZhERkbUtlDWQDUuPtmd28DlgGL4sosAh4Pl58BLrXgIQaLgGXu3uruW4AaYIEHmsLyOeHLw30+EdZBWOdVqTqwgZwwsYiS/GDezX1NbdQeOJyupoiIDJtUBsp0YFvM+9pwXcIy7h4FGoDy/vY1s4iZVQN7gN+7+4pwn/qwjr4+i3D/m8xstZmtTtX01llZ1usGRw3Mi8hYkMpASfS4tPhzP32V6XNfd+9w9ypgBrDAzM4Y5GcR7v+wu8939/mTJk3qs/HHam5FWfdy9YcKFBEZ/VIZKLVARcz7GcCOvsqYWTZQBuwfzL7uXg/8kWCMZR8wLqyjr88aVlUV47uX1UMRkbEglYGyCpgdXn2VSzDIvjyuzHLg+nD5GuBlD0awlwOLw6vAKoHZwEozm2Rm4wDMrAD4G+C9cJ9XwjoI6/x1Co9tQLE9lLXbG2jv6Exja0REUi9lgRKOZ9wKvAC8Czzt7uvN7E4z+0xY7BGg3MxqgC8DS8J91wNPAxuA3wG3uHsHMBV4xczeIQis37v7v4d1fRX4clhXeVh32kwuyWf6uAIAWto7+etuzTwsIqNbSh8B7O7PA8/HrftGzHILcG0f+94F3BW37h1gXh/lNxNcWZYx5laUsb0+uMKrels9p08rG2APEZGRS3fKp5DumBeRsUSBkkJ6NoqIjCUKlBQ6c0YZ4cTDbNrTRFNrtP8dRERGMAVKChXmZnPylBIA3OEdXT4sIqOYAiXF5s2MHUdpSGNLRERSS4GSYr3HUQ6ksSUiIqmlQEmxKvVQRGSMUKCk2OzJJRTmRgDYdbCFXQ0taW6RiEhqKFBSLJJlnDk9ZqJIXT4sIqOUAmUYVOkJjiIyBihQhoHumBeRsUCBMgxinzH/Tm09HZ16JLCIjD4KlGEwtSyfySV5ADS3dfD+3qYB9hARGXkUKMPAzHr1UvQERxEZjRQow6TXwLymYBGRUUiBMkyq1EMRkVFOgTJMzpxRhoUzD2/c3cjhto70NkhEJMkUKMOkND+HEycVA9DR6azboWlYRGR0UaAMI92PIiKjWUoDxcwWmtlGM6sxsyUJtueZ2VPh9hVmNitm29Jw/UYzuyxcV2Fmr5jZu2a23sxuiyn/TTPbbmbV4euKVB7bUMRe6fUXBYqIjDLZqarYzCLAA8AngVpglZktd/cNMcVuBA64+0lmthj4NnCdmc0BFgOnA9OAP5jZyUAU+Bd3f8vMSoA1Zvb7mDrvdffvpuqYjtU89VBEZBRLZQ9lAVDj7pvdvQ1YBiyKK7MIeDxcfga41MwsXL/M3VvdfQtQAyxw953u/haAuzcC7wLTU3gMSXXKcSXkZQdfee2Bw+xrak1zi0REkieVgTId2BbzvpYjf/l3l3H3KNAAlA9m3/D02DxgRczqW83sHTN71MzGJ2qUmd1kZqvNbPXevXuP9piOSU4kizNiZh5WL0VERpNUBoolWBc/iVVfZfrd18yKgV8Ct7v7wXD1g8CJQBWwE/i3RI1y94fdfb67z580aVL/R5ACvZ/gqEARkdEjlYFSC1TEvJ8B7OirjJllA2XA/v72NbMcgjB50t2f7Srg7rvdvcPdO4EfEZxyyzixT3BUoIjIaJLKQFkFzDazSjPLJRhkXx5XZjlwfbh8DfCyu3u4fnF4FVglMBtYGY6vPAK86+73xFZkZlNj3l4NrEv6ESVB1YzeA/OdmnlYREaJlF3l5e5RM7sVeAGIAI+6+3ozuxNY7e7LCcLhCTOrIeiZLA73XW9mTwMbCK7susXdO8zso8AXgbVmVh1+1L+6+/PA/zGzKoJTY1uB/5qqYzsWFRMKmFCUy/7mNg62RNla18wJ4Q2PIiIjWcoCBSD8Rf983LpvxCy3ANf2se9dwF1x614j8fgK7v7FY23vcDAz5s4o45WNwQUB1dvqFSgiMiroTvk0qKrouQBNV3qJyGihQEmDuRU9lw5rYF5ERgsFShrEzum1YedBWqOaeVhERj4FShqMK8xlVnkhAO0dzoYdBwfYQ0Qk8ylQ0kQzD4vIaKNASZNez5hXoIjIKKBASZNePZRaPWxLREY+BUqanDa1lJxIcEvNln3N1B9qS3OLRESOjQIlTfJzIsyZWtr9Xr0UERnpFChp1Gsc5UONo4jIyKZASaPe4ygKFBEZ2RQoaRR/pVcw0bKIyMikQEmjyvIiSvOD+Tn3N7dRe+BwmlskIjJ0CpQ0ysqyXr2Uv+h+FBEZwQYVKGZ2m5mVWuARM3vLzD6V6saNBbpjXkRGi8H2UP4xfHb7p4BJwD8Ad6esVWOInjEvIqPFYAOl66FWVwCPufvb9PGgKzk6sae81m1voL2jM42tEREZusEGyhoze5EgUF4wsxJAv/mSYFJJHtPHFQDQGu1k467GNLdIRGRoBhsoNwJLgHPc/RCQQ3DaS5KgaqZOe4nIyDfYQDkf2Oju9Wb2BeBrgOYKSZIqjaOIyCgw2EB5EDhkZnOB/wF8APx0oJ3MbKGZbTSzGjNbkmB7npk9FW5fYWazYrYtDddvNLPLwnUVZvaKmb1rZuvN7LaY8hPM7Pdmtin8OT7+8zJVbA9FV3qJyEg12ECJenAb9yLge+7+PaCkvx3MLAI8AFwOzAE+Z2Zz4ordCBxw95OAe4Fvh/vOARYDpwMLgR+E9UWBf3H304DzgFti6lwCvOTus4GXwvcjwhnTyohkBdc41OxtorGlPc0tEhE5eoMNlEYzWwp8EfhN+Ms9Z4B9FgA17r7Z3duAZQSBFGsR8Hi4/AxwqZlZuH6Zu7e6+xagBljg7jvd/S0Ad28E3gWmJ6jrceCqQR5b2hXkRjhlSpDP7rBWMw+LyAg02EC5DmgluB9lF8Ev8e8MsM90YFvM+1p6fvkfUcbdowTjMuWD2Tc8PTYPWBGumuLuO8O6dgKTEzXKzG4ys9Vmtnrv3r0DHMLw0R3zIjLSDSpQwhB5EigzsyuBFncfaAwl0X0q8bMf9lWm333NrBj4JXB7eMPloLn7w+4+393nT5o06Wh2Tal5umNeREa4wU698llgJXAt8FlghZldM8ButUBFzPsZwI6+yphZNlAG7O9vXzPLIQiTJ9392Zgyu81salhmKrBnMMeWKTTzsIiMdIM95fU/Ce5Bud7d/55gfOTrA+yzCphtZpVmlkswyL48rsxy4Ppw+Rrg5XDwfzmwOLwKrBKYDawMx1ceAd5193v6qet64NeDPLaMcNLkYopyIwDsaWxl18GWNLdIROToDDZQstw99n/8dQPtG46J3Aq8QDB4/rS7rzezO83sM2GxR4ByM6sBvkx4ZZa7rweeBjYAvwNucfcO4EKCCwM+YWbV4euKsK67gU+a2Sbgk4ywucYiWcaZM8q63+sJjiIy0mQPstzvzOwF4Ofh++uA5wfayd2fjy/n7t+IWW4hOI2WaN+7gLvi1r1GH3OIuXsdcOlAbcpkVRXjeXPzfgCqa+u5/MypaW6RiMjgDSpQ3P0rZvZ3BD0EAx529+dS2rIxqKpCPRQRGbkG20PB3X9JMBguKVJV0XNz/9rtDXR0evcNjyIima7fcRAzazSzgwlejWZ2VJfrysCOK8tnSmkeAIfaOti0RzMPi8jIMdDAeom7lyZ4lbh76XA1cizRExxFZKTSM+UzTPz9KCIiI4UCJcNU9QoUzeklIiOHAiXDnDm9DAvH4TfuOsihtmh6GyQiMkgKlAxTkp/D7MnFAHQ6rNuuax9EZGRQoGSgub2e4HggjS0RERk8BUoG6v0ER42jiMjIoEDJQHP1jHkRGYEUKBnolONKyM8J/mi21x9mT6NmHhaRzKdAyUA5kSzOmNYzr5dOe4nISKBAyVC6Y15ERhoFSobSHfMiMtIoUDJUrx5KbT2dnXoksIhkNgVKhpoxvoDyolwAGluibN7XnOYWiYj0T4GSocwsbl4vnfYSkcymQMlgczUwLyIjSEoDxcwWmtlGM6sxsyUJtueZ2VPh9hVmNitm29Jw/UYzuyxm/aNmtsfM1sXV9U0z225m1eHrilQe23BQD0VERpKUBYqZRYAHgMuBOcDnzGxOXLEbgQPufhJwL/DtcN85wGLgdGAh8IOwPoCfhOsSudfdq8LX88k8nnSIvWP+3Z0HaWnvSGNrRET6l8oeygKgxt03u3sbsAxYFFdmEfB4uPwMcKmZWbh+mbu3uvsWoCasD3d/FdifwnZnjLLCHE6YWARAtNNZv0MzD4tI5kploEwHtsW8rw3XJSzj7lGgASgf5L6J3Gpm74SnxcYnKmBmN5nZajNbvXfv3sEdSRppHEVERopUBoolWBd/M0VfZQazb7wHgROBKmAn8G+JCrn7w+4+393nT5o0aYAq00/jKCIyUqQyUGqBipj3M4AdfZUxs2ygjOB01mD27cXdd7t7h7t3Aj8iPEU20s2Nu8FRRCRTpTJQVgGzzazSzHIJBtmXx5VZDlwfLl8DvOzuHq5fHF4FVgnMBlb292FmNjXm7dXAur7KjiSnTS0hNxL8MX1Qd4j9zW1pbpGISGIpC5RwTORW4AXgXeBpd19vZnea2WfCYo8A5WZWA3wZWBLuux54GtgA/A64xd07AMzs58AbwClmVmtmN4Z1/R8zW2tm7wAfB/45Vcc2nPKyI5w2rbT7vXopIpKpslNZeXjp7vNx674Rs9wCXNvHvncBdyVY/7k+yn/xmBqbweZVjOsekK/+sJ6PnzI5zS0SETmS7pQfAeZWxDwbRT0UEclQCpQRoKqi5wrot7fVEwwziYhkFgXKCDCrvJCyghwADhxq58P9h9LcIhGRIylQRgAz0wO3RCTjKVBGiKoZPeMoChQRyUQKlBGiaqZ6KCKS2RQoI0TszMPrdxykLdqZxtaIiBxJgTJClBfnUTGhAIC2aCfv7dLMwyKSWRQoI0hsL0UzD4tIplGgjCCxMw//RYEiIhlGgTKCVOnZKCKSwRQoI8gZ08uIZAWPinl/bzMNh9vT3CIRkR4KlBEkPyfCqceVdL9fW9uQxtaIiPSmQBlhej/B8UAaWyIi0psCZYTpPQWLeigikjkUKCPMvLg5vTTzsIhkCgXKCHPCpGKK84Lnou1ramVHQ0uaWyQiElCgDMXBHXBwZ1o+OpJlnBU7UeSHunxYRDKDAmUoXrkLvncW/L/b4cDWYf/42HEUPcFRRDJFSgPFzBaa2UYzqzGzJQm255nZU+H2FWY2K2bb0nD9RjO7LGb9o2a2x8zWxdU1wcx+b2abwp/jSZWL7oCq/wzVT8L9H4Hnboa9f03Zx8XrdaWXeigikiFSFihmFgEeAC4H5gCfM7M5ccVuBA64+0nAvcC3w33nAIuB04GFwA/C+gB+Eq6LtwR4yd1nAy+F71NjQiX8p/vgtrfh3Jth/a/ggQXw9N/DzrdT9rFdYgNl7fYGoh2aeVhE0i+VPZQFQI27b3b3NmAZsCiuzCLg8XD5GeBSM7Nw/TJ3b3X3LUBNWB/u/iqwP8Hnxdb1OHBVMg8modJpsPB/wz+vg4u+DO+/Aj+8GJ68Fj5ckbKPnVKaz9SyfAAOt3fw191NKfssEZHBSmWgTAe2xbyvDdclLOPuUaABKB/kvvGmuPvOsK6dwOREhczsJjNbbWar9+7dO8hDGUDRRLj0G3D7WvjE16B2NTz6KfjJlUHIpODS3tiZh2/+2RqeePMDWto7kv45IiKDlcpAsQTr4n+z9lVmMPsOibs/7O7z3X3+pEmTklFlj4JxcPFXgh7LZf8b6mrgiavgx5fCe88nNVguPKm8e/nD/Yf4+q/WceHdL3P/S5s40NyWtM8RERmsVAZKLVAR834GsKOvMmaWDZQRnM4azL7xdpvZ1LCuqcCeIbf8WOUWwfm3BGMsV94Lzftg2efgwQth7TPQeew9icULZvLlT55MWUFO97q65jbu+f1fueDul/nm8vXUHjh0zJ8jIjJYqQyUVcBsM6s0s1yCQfblcWWWA9eHy9cAL3tw6/dyYHF4FVglMBtYOcDnxdZ1PfDrJBzDscnOg/n/CP/9Lbj6h9AZhV/eCN8/B956AqJD70nkRLL4p0tn8/qST/CNK+cwfVxB97bD7R385PWtXPKdP3L7sr+wYYee7igiqWepnLrDzK4A7gMiwKPufpeZ3QmsdvflZpYPPAHMI+iZLHb3zeG+/xP4RyAK3O7uvw3X/xz4GDAR2A38L3d/xMzKgaeBmcCHwLXunmjwvtv8+fN99erVyT7svnV2wnv/D179Lux6B8oq4MLbYN4XIKdg4P370d7RyW/e2clDf3qf93Y1HrH94pMncfPFJ3D+ieUE1z2IiAyNma1x9/lHrB/Lc0ENe6B0cYeaPwTBsu1NKJoMF9wa9GbySgbev9+qnVc37eOhP77PG5vrjth+5vQy/uslJ7Dw9OPIjui+VhE5egqUBNIWKF3c4YP/CIJl8yuQPw7O+2+w4CYonHDM1b+9rZ6HX93Mb9ftpDPuj3nmhEL+y0WVXHN2BQW5kcQViIgkoEBJIO2BEqt2Dfz5u7DxecgthnNuhPNvheKEVz8fla37mvnxa5v5xepaWqO9b4KcUJTL9efP4u/PP57xRbnH/FkiMvopUBLIqEDpsns9/PnfYP1zEMmFj/w9XPBPMK5i4H0HsMQFx3kAABIjSURBVK+plZ++vpXH3/jgiMcHF+REuO6cCm78aCUVEwqP+bNEZPRSoCSQkYHSpe59eO0eeHtZ8H7uYvjol6H8xGOuurk1ylOrtvHIa1vYXn+417ZIlnHlWVO56eITOH1aWR81iMhYpkBJIKMDpUv9Nnj9fnjrp9DRBqdfDRf9C0w5/ZirHujKsItmT+TmS07kAl0ZJiIxFCgJjIhA6dK0B974Pqx6BNqa4JQrglmPZ5x9zFV3XRn2wz+9z+vv68owEemfAiWBERUoXQ7th5UPw5sPQks9lE4PbqCM5EIkJ/w5lOXgVdsY5Y819aypbabNs2knQhvZtJPNhJJirqiaycdPn0F+Xn5PHdl5UDQJsnS1mMhYoEBJYEQGSpfWRljzOOx5NzgV1tEGHe39LLf2vT0Z8spg5rlw/AUw8wKYNg+yddWYyGjUV6Bkp6MxkgR5JcHNkMfKPZgSJlHQRNs40NTEb//yIS+u3UZrWyu5RMkJX0XZnXy0spSLK0uY0PgefPAGbHoxqDe7AGbMDwLm+AtgxjnBHGciMmqphzJSeyjDrLk1ytOrt/HjPye+MuySkydx3gkTuOA459S29WTXvhnctLlrLXgnZGXD1KowYC4MejMFqXuopoikjk55JaBAOXrtHZ08v3YnD/1pM+/uTDzpZEFOhLOPH885syZw3vRsqvgredvfhA9eh+1roLMdsOBKta4ezMwLoGTK8B6MiAyJAiUBBcrQDXRlWKyciHHm9DIWVJZzXkUB83O2ULxrZdCD2bYK2puDghNO7AmY4y+AcceDLlcWyTgKlAQUKMmxZV8zb26uY+WW/azcsv+IU2LxzODU40pZMGs85x5fynmFtUzYuxo+fCPoxbTUBwVLpvUOmImnQJYuWxZJNwVKAgqU1Nhef5hVW/azcmsQMDV7Bn7mfeXEIs6ZNZ5zjh/HR0v3cVzDW9gHrwcB07QrKFQwIeYU2flw3FkQ0XUlIsNNgZKAAmV41DW1smrrAVaFAbN+R8MRsx/Hm1KaxzmzJnDurPFcUN5IZfPbZHX1YA5sCQrlFkPFuXD8+cFA/7SPQE5+6g9IZIxToCSgQEmPxpZ23vqwvrsXU72tnra4WZDjlRXkcM6s8SyonMAFk9o4tW1deCXZ67BnQ1AokhuMu4w/Pvg5bmbM8vHBIwE0JiNyzBQoCShQMkNLewdrtzd0j8Gs+eAATa3RfvcpyInwkePHcc6sCVwwzZjbuZG8natg/2ao/xDqP4DDB3rvlFschEx36MQt52syTJHBUKAkoEDJTNGOTt7b1ciKLftZtWU/q7bup665/zv6cyLGGdPLOHlyCVPH5TNtXAEzC6PMyNrH5Oguchu39QRN/Ydw4ANoi5sQM39c76Dp1dupyKwbM6OtwWwJrQeDny3hz9h1HW2QUwh5xZBbErQ/rzj42et9sabNkaOiQElAgTIyuDvv723uHoMZzJVk8cqLcpk2roBpYdhML8tnZkErMyP7mNq5m5KWHWQ1hEHTFTrRlt6VFE1KcCptJoyfBWUzgjnNBhJtDX/5xwZATAjErk8UEl3LyZoyp0t2Qe+w6V4OA6creHKLglkaurZ1r4/dVqxpd0Y5BUoCCpSRayhXkvUnJ2IcV5bPtLICpo8rYFpZPpWFzcyK7GOa72FC+07yGmuDsDnwATTUhjdodjEomRoETclx0N4S/vJv6B0cgwmCrBzILw1+OeeVQF5p+CqJe5XGlSuJKVcaTNzZfgham4IZqtuawuXmcLmxZ7nfbc3htsZg1oPByMrpCZv8MiiaCIUTg1AuKg9+Fk4M1hdNgsLyoJzGuEaEtASKmS0EvgdEgB+7+91x2/OAnwJnA3XAde6+Ndy2FLgR6AD+yd1f6K9OM/sJcAnQEFZ/g7tX99c+BcroUdfUytu19Ww/cJjt9S3sbDjMjvrD7KhvYdfBFjoGuqxsEErzs8NeTgHTy3KYXdBEZaSOabaHie27KDm8nayGbcFlzjmFvUPgiIAoSRAS4ftMvVLNPei1xQZUbNh0L8cFVEs9NO+D5r1wqC7oYSWSlRMGTGz4JHhfGAZSXkl6A8gd2g/3Dt6Ey3HvLQsieT2zfWfnJZz5O9gWOzN43KzivfaLqyPF92sNe6CYWQT4K/BJoBZYBXzO3TfElPn/gLPc/WYzWwxc7e7Xmdkc4OfAAmAa8Afg5HC3hHWGgfLv7v7MYNuoQBkboh2d7G1qZUd9EDY76g+zM2Z5R8Nh6g+1D1zRALIMppTmM7Usn+PK8plcks+U0nymlOZ1/5xcmk9JXvbYfmBZe0sQLM174dC+MGz2hct7oTluW1sfvc9Ibk/vpjt8+no/MfhFnugXfmt/YZDoZ8z2wfbYLCvoreUUBEEUOwt4Z/8XoAxJVnaCgMrpHUIL74aKBUOqPh2zDS8Aatx9c9iAZcAiYENMmUXAN8PlZ4DvW/AvbRGwzN1bgS1mVhPWxyDqFOklO5LF1LICppYVcPbxicscaouyoytg6g+zo6FneWdDC9vrDw94aXOnw86GFnY2tPRbrjA3wpTSfCaX5PUKnMml+UzpXpdPQe4oHSjPyYey6cFrMNoPxwRObNjEhU/dpuB911Q+Q2VZPRctdL+Kofi4uLGlorjlfrblFPTdm+rsDE6fhjN898z83dZr5u8j13ftE/toitY+1ieoK5JzbN9TAqkMlOnAtpj3tcC5fZVx96iZNQDl4fo34/bt+tvXX513mdk3gJeAJWEg9WJmNwE3AcycOfMoD0lGq8LcbE6aXMxJk4sTbnd36prbuk+j9QRPz/s9jUf8dUvoUFsHW/Y1s2Vf/7/4SvOzu8NlclcvpyQmfErzmFyST272KJ+OJqcguMpuXMXgyrcdStzzcY+7qCDRL//i4H/xw9mDzMqCrLzgcwdxXUcmS2WgJPoTiT+/1leZvtYn+pfTVedSYBeQCzwMfBW484jC7g+H25k/f/7YvSJBjoqZMbE4j4nFeZw1I3GZ1mgHuxta2dFwmN0HW9hzsJXdB1vY3dgavg/Gc1raB3ea5GBLlIMtTWwa4IKD8qLc7oCZUtJzam1ySR7lxXlMLM6lvDiPotzI2DjVllsIuTODK/BkWKUyUGqB2P9SzAB29FGm1syygTJg/wD7Jlzv7jvDda1m9hhwRxKOQWTQ8rIjzCwvZGZ5YZ9l3J3G1mgQLg1dgRMTPgdb2H2wlT2NLbR3DO7/O3XNbdQ1t/Huzv7L5WZnMbEoCJcJRbmUF+cysTiP8qJcJhSFy8U9y/k5o/SUm6RMKgNlFTDbzCqB7cBi4PNxZZYD1wNvANcAL7u7m9ly4P+a2T0Eg/KzgZUEPZeEdZrZVHffGY7BXAWsS+GxiQyJmVGan0Npfg4nTS7ps1xnp3PgUBu7D7aGgRMETWzg7GpoYV9T64DzonVpi3YGY0MDjPF0KcqNMKE4l/KisJdTlBe+DwInNpTGF+aO/lNvMqCUBUo4JnIr8ALBJb6Puvt6M7sTWO3uy4FHgCfCQff9BAFBWO5pgsH2KHCLu3cAJKoz/MgnzWwSQehUAzen6thEUi0ryygvDk5ZzaG0z3LRjk7qmtu6g6br1FpXEO1vbqOuqY19Ta20DnBRQbzmtg6a9x9m2/7B3URamp/dq5dTXpzHxKJcxhXmUpyfTXFe+MrPpiQvm6JwuSg3m0jWGDgVNwboxkZdNixjgLtzqK2D/c1BuNQ1tQXLzTHLMevrmlsHfcotGYpyI90BUxITNL3e5/UsF+flUJQXoSQvp1dY5edkjY1xojRLx2XDIpIhzIyisFdQMaHvMZ4u7s7Blih1Ta1h2AQhs78pGK/ZF66v61rf3DboU2+JNLd10NzWMegr5foSybKenlB3+AQ/ywpymFCYy/ii4LRd7M8Jhbmj9zLtYaRAEZEjmBllBTmUFeRwwqSBy3d2OvWH26lrag0uEgiDZl9TGwcPt9PYEqW5NUpTa5TG1ihNLe00tUZpbu0YcGbpo9HR6TQcbqfh8NHfqFqQE2FCeIFCEDI5TCjKY0JRz8/xhcG40fjC4FSeTtX1pkARkWOWlWXdv4xnH+W+nZ1Oc1s0DJgojS3BclNLED7N4XJXGDXHbOta3xxuG+jm0/4cbu9ge/3hQU88agbjCnJ6ejqFud3fQe9g6nlfOMov3VagiEhaZWUZJfk5lOQf+53brdEOmls7egdTa9BDajjczv7mtiNeBw4FP492zMgdDhxq58ChdjbvHdzd+bnZWeRlZxHJMiJmmBmRLGKWg1eWQZZ1LRtZYZmscL/udV3bY8tauL67LHH79Xzmv376NPKyk3eqT4EiIqNGXnaEvOzg1NXRcHeaWqO9QqauqSts2tnf3Nr988ChIJiGclqtLdp5TL2oZFt6xWlJrU+BIiJjnllPL+n48sE9SK29o5P6Q+29Q6i5jQNxvZ+uYKprbsuoMIGgF5RMChQRkSHIiWQxqSSPSSWDm4DL3Tnc3kF71Olwp6PTcY9dDi4q6HCns9PpDN93evDqWu7oJFjXVdaDcaiufT0s07PsMfXE1NnpZCf5ogIFiojIMDAzCnOzg9kGRynNlSAiIkmhQBERkaRQoIiISFIoUEREJCkUKCIikhQKFBERSQoFioiIJMWYfh6Kme0FPkh3O47RRGBfuhuRQfR99NB30Zu+j96O5fs43t2PmId6TAfKaGBmqxM96Gas0vfRQ99Fb/o+ekvF96FTXiIikhQKFBERSQoFysj3cLobkGH0ffTQd9Gbvo/ekv59aAxFRESSQj0UERFJCgWKiIgkhQJlhDKzCjN7xczeNbP1ZnZbutuUbmYWMbO/mNm/p7st6WZm48zsGTN7L/w7cn6625QuZvbP4b+RdWb2czPLT3ebhpOZPWpme8xsXcy6CWb2ezPbFP4cn4zPUqCMXFHgX9z9NOA84BYzm5PmNqXbbcC76W5Ehvge8Dt3PxWYyxj9XsxsOvBPwHx3PwOIAIvT26ph9xNgYdy6JcBL7j4beCl8f8wUKCOUu+9097fC5UaCXxjT09uq9DGzGcCngR+nuy3pZmalwMXAIwDu3ubu9eltVVplAwVmlg0UAjvS3J5h5e6vAvvjVi8CHg+XHweuSsZnKVBGATObBcwDVqS3JWl1H/A/gM50NyQDnADsBR4LTwH+2MyK0t2odHD37cB3gQ+BnUCDu7+Y3lZlhCnuvhOC/5wCk5NRqQJlhDOzYuCXwO3ufjDd7UkHM7sS2OPua9LdlgyRDXwEeNDd5wHNJOmUxkgTjg0sAiqBaUCRmX0hva0avRQoI5iZ5RCEyZPu/my625NGFwKfMbOtwDLgE2b2s/Q2Ka1qgVp37+qxPkMQMGPR3wBb3H2vu7cDzwIXpLlNmWC3mU0FCH/uSUalCpQRysyM4Bz5u+5+T7rbk07uvtTdZ7j7LIIB15fdfcz+L9TddwHbzOyUcNWlwIY0NimdPgTOM7PC8N/MpYzRCxTiLAeuD5evB36djEqzk1GJpMWFwBeBtWZWHa77V3d/Po1tkszx34EnzSwX2Az8Q5rbkxbuvsLMngHeIrgy8i+MsSlYzOznwMeAiWZWC/wv4G7gaTO7kSB0r03KZ2nqFRERSQad8hIRkaRQoIiISFIoUEREJCkUKCIikhQKFBERSQoFisgIZWYf08zKkkkUKCIikhQKFJEUM7MvmNlKM6s2sx+Gz21pMrN/M7O3zOwlM5sUlq0yszfN7B0ze67rORVmdpKZ/cHM3g73OTGsvjjmuSdPhneDi6SFAkUkhczsNOA64EJ3rwI6gP8MFAFvuftHgD8R3L0M8FPgq+5+FrA2Zv2TwAPuPpdgLqqd4fp5wO3AHIJZhi9M+UGJ9EFTr4ik1qXA2cCqsPNQQDARXyfwVFjmZ8CzZlYGjHP3P4XrHwd+YWYlwHR3fw7A3VsAwvpWuntt+L4amAW8lvrDEjmSAkUktQx43N2X9lpp9vW4cv3NgdTfaazWmOUO9G9a0kinvERS6yXgGjObDN3P8j6e4N/eNWGZzwOvuXsDcMDMLgrXfxH4U/icm1ozuyqsI8/MCof1KEQGQf+bEUkhd99gZl8DXjSzLKAduIXgoVenm9kaoIFgnAWCqcQfCgMjdpbgLwI/NLM7wzqSMjusSDJptmGRNDCzJncvTnc7RJJJp7xERCQp1EMREZGkUA9FRESSQoEiIiJJoUAREZGkUKCIiEhSKFBERCQp/n8Tz7MHZddICQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "mnist_train = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=True, download=True, transform=transforms.ToTensor())\n",
    "mnist_test = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=False, download=True, transform=transforms.ToTensor())\n",
    "\n",
    "batch_size = 200\n",
    "train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False,num_workers=0)\n",
    "\n",
    "num_inputs, num_outputs, num_hiddens1, num_hiddens2 = 784, 10, 256, 256\n",
    "\n",
    "W1 = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_hiddens1)), dtype=torch.float, requires_grad=True)\n",
    "b1 = torch.zeros(num_hiddens1,dtype=torch.float, requires_grad=True)\n",
    "W2 = torch.tensor(np.random.normal(0, 0.01, (num_hiddens1, num_hiddens2)), dtype=torch.float, requires_grad=True)\n",
    "b2 = torch.zeros(num_hiddens2,dtype=torch.float,requires_grad=True)\n",
    "W3 = torch.tensor(np.random.normal(0, 0.01, (num_hiddens2, num_outputs)), dtype=torch.float, requires_grad=True)\n",
    "b3 = torch.zeros(num_outputs,dtype=torch.float,requires_grad=True)\n",
    "\n",
    "params = [W1, b1, W2, b2, W3, b3]\n",
    "\n",
    "def dropout(X, drop_prob):\n",
    "    X = X.float()\n",
    "    assert 0 <= drop_prob <=1\n",
    "    keep_prob = 1 - drop_prob\n",
    "    if keep_prob == 0:\n",
    "        return torch.zeros_like(X)\n",
    "    mask = (torch.randn(X.shape) < keep_prob).float()\n",
    "    return mask * X / keep_prob\n",
    "\n",
    "drop_prob1, drop_prob2 = 0.3, 0.5\n",
    "\n",
    "def net(X, is_training = True):\n",
    "    X = X.view(-1, num_inputs)\n",
    "    H1 = (torch.matmul(X, W1) + b1).relu()\n",
    "    if is_training:\n",
    "        H1 = dropout(H1, drop_prob1)\n",
    "    H2 = (torch.matmul(H1, W2) + b2).relu()\n",
    "    if is_training:\n",
    "        H2 = dropout(H2, drop_prob2)\n",
    "    return torch.matmul(H2, W3) + b3\n",
    "\n",
    "def sgd(params, lr, batch_size):\n",
    "    for param in params:\n",
    "        param.data -= lr * param.grad / batch_size\n",
    "\n",
    "def evaluate_accuracy(data_iter, net):    \n",
    "    acc_sum, n = 0.0, 0\n",
    "    for X, y in data_iter:\n",
    "        if isinstance(net, torch.nn.Module):  \n",
    "            net.eval()  #api文档：将模型设置成evaluation模式，仅仅当模型中有Dropout和BatchNorm是才会有影响。\n",
    "            acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()\n",
    "            net.train()\n",
    "        else:\n",
    "            if('is_training' in net.__code__.co_varnames):\n",
    "                acc_sum += (net(X, is_training=False).argmax(dim=1) == y).float().sum().item()\n",
    "            else:\n",
    "                acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()\n",
    "        n += y.shape[0]\n",
    "    return acc_sum / n\n",
    "\n",
    "num_epochs, lr = 10, 106.0\n",
    "loss = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, params , lr):\n",
    "    train_ls, test_ls, x_epoch = [], [], []\n",
    "    for epoch in range(num_epochs):\n",
    "        train_1_sum = 0.0\n",
    "        train_1_test_sum = 0.0\n",
    "        train_acc_sum = 0.0\n",
    "        n = 0\n",
    "        n_test = 0\n",
    "        for X, y in train_iter:\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat, y).sum()\n",
    "            if params[0].grad is not None:\n",
    "                for param in params:\n",
    "                    param.grad.data.zero_()\n",
    "\n",
    "            l.backward()\n",
    "            sgd(params, lr, batch_size)\n",
    "\n",
    "            train_1_sum += l.item()\n",
    "            train_acc_sum += (y_hat.argmax(dim =1) == y).sum().item()\n",
    "            n += y.shape[0]\n",
    "        train_ls.append(train_1_sum / n)\n",
    "        x_epoch.append(epoch + 1)\n",
    "        test_acc = evaluate_accuracy(test_iter, net)\n",
    "        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f' % (epoch + 1, train_1_sum / n, train_acc_sum / n, test_acc))\n",
    "\n",
    "        for X_test, y_test in test_iter:\n",
    "            y_hat = net(X_test)\n",
    "            l = loss(y_hat, y_test).sum()\n",
    "            train_1_test_sum += l.item()\n",
    "            n_test += y_test.shape[0]\n",
    "        test_ls.append(train_1_test_sum / n_test)\n",
    "\n",
    "\n",
    "    plt.plot(x_epoch, train_ls, label=\"train_loss\", linewidth=3)\n",
    "    plt.plot(x_epoch, test_ls, label=\"test_loss\", linewidth=1.5)\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.ylabel(\"loss\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, params, lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, loss 0.0085, train acc 0.441, test acc 0.816\n",
      "epoch 2, loss 0.0049, train acc 0.678, test acc 0.863\n",
      "epoch 3, loss 0.0039, train acc 0.748, test acc 0.883\n",
      "epoch 4, loss 0.0034, train acc 0.784, test acc 0.892\n",
      "epoch 5, loss 0.0031, train acc 0.806, test acc 0.902\n",
      "epoch 6, loss 0.0028, train acc 0.823, test acc 0.908\n",
      "epoch 7, loss 0.0027, train acc 0.836, test acc 0.912\n",
      "epoch 8, loss 0.0025, train acc 0.845, test acc 0.915\n",
      "epoch 9, loss 0.0024, train acc 0.851, test acc 0.918\n",
      "epoch 10, loss 0.0023, train acc 0.857, test acc 0.921\n",
      "epoch 11, loss 0.0022, train acc 0.865, test acc 0.924\n",
      "epoch 12, loss 0.0021, train acc 0.870, test acc 0.927\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3yV9d3/8dcnew+yIKwEGQoqQSKjKNZSFdSKW2xVrFp+3j9ttf21Fe7eXXTZ++7QPmq1VrG2WkdRlLsOcBZtZQSMsk3YIZBJFiH78/vjupIcQkLWOTkZn+fjkcc55xrf870Q8+a6vktUFWOMMaarAvxdAWOMMQOLBYcxxphuseAwxhjTLRYcxhhjusWCwxhjTLcE+bsCfSExMVHT0tL8XQ1jjBkwNm/eXKyqSe3tGxLBkZaWRlZWlr+rYYwxA4aIHOhonz2qMsYY0y0WHMYYY7rFgsMYY0y3DIk2DmPM4FNfX09eXh41NTX+rsqAFhYWxqhRowgODu7yORYcxpgBKS8vj+joaNLS0hARf1dnQFJVSkpKyMvLIz09vcvn2aMqY8yAVFNTQ0JCgoVGL4gICQkJ3b5rs+AwxgxYFhq915M/QwuODmzNK+e2FRv53qqt/q6KMcb0KxYcHQgOEtZ9VsQ7OwuxNUuMMaaVBUcHJiZHMywyhKMVNRwoqfZ3dYwx/UxZWRl/+MMfun3e5ZdfTllZWbfPu/3221m5cmW3z/MFC44OBAQIM9OHAfDR3hI/18YY0990FByNjY2nPe/1118nLi7OV9XqE9Yd9zRmn5HAG9uO8tGeEm6eMcbf1THGdCBt6Ws+KXf/g1d0uG/p0qXs2bOHjIwMgoODiYqKYsSIEWRnZ7Njxw6uvvpqDh06RE1NDffddx9Llixx6urOnVdVVcWCBQu44IIL+Pe//83IkSN59dVXCQ8P77Re77zzDt/+9rdpaGjg/PPP59FHHyU0NJSlS5eyevVqgoKCuPTSS/nVr37F3//+d3784x8TGBhIbGws69at6/WfiwXHacwelwA4dxyqaj04jDEtHnzwQbZt20Z2djbvv/8+V1xxBdu2bWsZD7FixQqGDRvGiRMnOP/887nuuutISEg4qYycnByee+45/vSnP3HjjTfy0ksvccstt5z2e2tqarj99tt55513mDhxIrfddhuPPvoot912G6tWrWLXrl2ISMvjsOXLl7NmzRpGjhzZo0dk7bHgOI3xyVEkRoVSVFnLnqLjjE+O8neVjDHtON2dQV+ZMWPGSYPofve737Fq1SoADh06RE5OzinBkZ6eTkZGBgDTp09n//79nX7P7t27SU9PZ+LEiQAsXryYRx55hHvvvZewsDDuuusurrjiCq688koA5syZw+23386NN97Itdde641LtTaO0xERZo2zdg5jTOciIyNb3r///vu8/fbbfPTRR3zyySdMmzat3UF2oaGhLe8DAwNpaGjo9Hs66uUZFBTExo0bue6663jllVeYP38+AI899hg//elPOXToEBkZGZSU9P53mQVHJ2af4fwLYf0eCw5jTKvo6GgqKyvb3VdeXk58fDwRERHs2rWL9evXe+17zzzzTPbv309ubi4Af/3rX7nooouoqqqivLycyy+/nIceeojs7GwA9uzZw8yZM1m+fDmJiYkcOnSo13WwR1WdaG7nWG/tHMYYDwkJCcyZM4ezzz6b8PBwUlJSWvbNnz+fxx57jHPPPZdJkyYxa9Ysr31vWFgYTz31FDfccENL4/jdd99NaWkpCxcupKamBlXlt7/9LQDf+c53yMnJQVWZN28eU6dO7XUdZCgMbsvMzNSergCoqsz6xTsUVNSy5v65TBoe7eXaGWN6YufOnZx11ln+rsag0N6fpYhsVtXM9o63R1WdEJHW3lV7iv1cG2OM8T8Lji5oaefYW+rnmhhjBrt77rmHjIyMk36eeuopf1frJNbG0QWzmts59pXQ1KQEBFg7hzHGNx555BF/V6FTPr3jEJH5IrJbRHJFZGk7+0NF5AV3/wYRSfPYt8zdvltELvPY/k0R2S4i20TkOREJ8+U1AIwZFkFqbBhl1fXsOtp+LwpjjBkqfBYcIhIIPAIsACYDN4vI5DaH3QkcU9XxwG+BX7rnTgYWAVOA+cAfRCRQREYC3wAyVfVsINA9zqdEhFlntI4iN8aYocyXdxwzgFxV3auqdcDzwMI2xywEnnbfrwTmidPfdSHwvKrWquo+INctD5zHa+EiEgREAPk+vIYWrQ3kFhzGmKHNl8ExEvAcaZLnbmv3GFVtAMqBhI7OVdXDwK+Ag8ARoFxV17b35SKyRESyRCSrqKio1xfT3EC+YV8JjU2DvwuzMcZ0xJfB0V4LctvfuB0d0+52EYnHuRtJB1KBSBFpd0YwVX1cVTNVNTMpKakb1W7fqPgIRg8Lp7KmgR35Fb0uzxgzsPV0PQ6Ahx56iOrq06/zk5aWRnFx/xwC4MvgyANGe3wexamPlVqOcR89xQKlpzn3i8A+VS1S1XrgZeBzPql9O1pny+2f/zGNMX3H18HRn/myO+4mYIKIpAOHcRqxv9zmmNXAYuAj4HrgXVVVEVkN/E1EfoNzZzEB2Ag0AbNEJAI4AcwDejYkvAdmn5HAi1l5fLSnhCVzz+irrzXGdOaNpXB0q3fLHH4OLHiww92e63FccsklJCcn8+KLL1JbW8s111zDj3/8Y44fP86NN95IXl4ejY2NfP/736egoID8/HwuvvhiEhMTee+99zqtym9+8xtWrFgBwF133cX999/fbtk33XRTu2tyeJvPgkNVG0TkXmANTu+nFaq6XUSWA1mquhp4EviriOTi3Gkscs/dLiIvAjuABuAeVW0ENojISmCLu/1j4HFfXUNbs8clArBp/zEaGpsICrTxk8YMVZ7rcaxdu5aVK1eyceNGVJWrrrqKdevWUVRURGpqKq+95iw0VV5eTmxsLL/5zW947733SExM7PR7Nm/ezFNPPcWGDRtQVWbOnMlFF13E3r17Tym7tLS03TU5vM2nAwBV9XXg9TbbfuDxvga4oYNzfwb8rJ3tPwR+6N2ads3w2DDSEyPZV3ycbfkVZIwe2Ms/GjNonObOoC+sXbuWtWvXMm3aNACqqqrIycnhwgsv5Nvf/jYPPPAAV155JRdeeGG3y/7www+55pprWqZtv/baa/nggw+YP3/+KWU3NDS0uyaHt9k/mbupZX0O65ZrjHGpKsuWLSM7O5vs7Gxyc3O58847mThxIps3b+acc85h2bJlLF++vEdlt6e9sjtak8PbLDi6adY4GwhojDl5PY7LLruMFStWUFVVBcDhw4cpLCwkPz+fiIgIbrnlFr797W+zZcuWU87tzNy5c3nllVeorq7m+PHjrFq1igsvvLDdsjtak8PbbK6qbmruWZW1v5T6xiaCrZ3DmCHJcz2OBQsW8OUvf5nZs2cDEBUVxTPPPENubi7f+c53CAgIIDg4mEcffRSAJUuWsGDBAkaMGNFp4/h5553H7bffzowZzhjou+66i2nTprFmzZpTyq6srGx3TQ5vs/U4emDer99nT9FxXvqP2UwfO8xr5Rpjus7W4/AeW4+jDzSPIrd2DmPMUGTB0QPN3XKtncMY01szZ848Zf2NrVu9PCbFy6yNoweae1Zl7T9GbUMjoUGBfq6RMUOTquLMizpwbdiwwa/f35PmCrvj6IGEqFAmpURT29BE9kHfDLAxxpxeWFgYJSUlPfrFZxyqSklJCWFh3VvWyO44emj2GQnsLqjko70lzHR7Whlj+s6oUaPIy8vDG7NfD2VhYWGMGjWqW+dYcPTQrHHD+PO/97Pe2jmM8Yvg4GDS09P9XY0hyR5V9dDM9AREYMvBMmrqG/1dHWOM6TMWHD0UHxnCmcNjqGtoYsvBY/6ujjHG9BkLjl5oHkW+3sZzGGOGEAuOXmgZCGjtHMaYIcSCoxdmpA8jQCD7UBkn6qydwxgzNFhw9EJseDBTUmOpb1SyDpT6uzrGGNMnLDh6yeatMsYMNRYcvTTb1ucwxgwxFhy9lJkWT2CA8GleOVW1Df6ujjHG+JwFRy9FhwVz9shYGpuUrP3WzmGMGfwsOLzAHlcZY4YSnwaHiMwXkd0ikisiS9vZHyoiL7j7N4hImse+Ze723SJymbttkohke/xUiMj9vryGrmhuILeBgMaYocBnkxyKSCDwCHAJkAdsEpHVqrrD47A7gWOqOl5EFgG/BG4SkcnAImAKkAq8LSITVXU3kOFR/mFgla+uoasyx8YTFCBsPVxORU09MWHB/q6SMcb4jC/vOGYAuaq6V1XrgOeBhW2OWQg87b5fCcwTZ1WWhcDzqlqrqvuAXLc8T/OAPap6wGdX0EWRoUFMHR1Hk8KmfdbOYYwZ3HwZHCOBQx6f89xt7R6jqg1AOZDQxXMXAc919OUiskREskQkqy/m629p57DHVcaYQc6XwdHeeo5tl+rq6JjTnisiIcBVwN87+nJVfVxVM1U1MykpqQvV7R2bt8oYM1T4MjjygNEen0cB+R0dIyJBQCxQ2oVzFwBbVLXAy3Xuselj4wkJDGDHkQrKquv8XR1jjPEZXwbHJmCCiKS7dwiLgNVtjlkNLHbfXw+8q84CwquBRW6vq3RgArDR47ybOc1jKn8ICw4kY0wcqrDB2jmMMYOYz4LDbbO4F1gD7AReVNXtIrJcRK5yD3sSSBCRXOBbwFL33O3Ai8AO4E3gHlVtBBCRCJyeWi/7qu49Ze0cxpihwKdrjqvq68Drbbb9wON9DXBDB+f+DPhZO9urcRrQ+51Z4xJ4+J0cW4fcGDOo2chxL5o2Jo6QoAB2Ha2k9Li1cxhjBicLDi8KCw5k+ph4ADbYXYcxZpCy4PAy65ZrjBnsLDi8zBZ2MsYMdhYcXjZ1VBzhwYHkFFZRVFnr7+oYY4zXWXB4WUhQAJlpTjuH9a4yxgxGFhw+MMvW5zDGDGIWHD5g63MYYwYzCw4fOGdkLBEhgewtPk5BRY2/q2OMMV5lweEDwYEBnJ82DLB2DmPM4GPB4SPWLdcYM1hZcPjIbGsgN8YMUhYcPjIlNYbo0CAOlFSTX3bC39UxxhivseDwkaDAAGakO+0c9rjKGDOYWHD4kM1bZYwZjCw4fGiWLexkjBmELDh8aPKIGGLDgzlcdoJDpdX+ro4xxniFBYcPBQQIM62dwxgzyFhw+JjNW2WMGWwsOHzMcyCgqvq5NsYY03sWHD42KSWa+IhgjlbUcKDE2jmMMQOfBYePBQSIPa4yxgwqPg0OEZkvIrtFJFdElrazP1REXnD3bxCRNI99y9ztu0XkMo/tcSKyUkR2ichOEZnty2vwBpu3yhgzmPgsOEQkEHgEWABMBm4WkcltDrsTOKaq44HfAr90z50MLAKmAPOBP7jlATwMvKmqZwJTgZ2+ugZv8Zy3yto5jDEDnS/vOGYAuaq6V1XrgOeBhW2OWQg87b5fCcwTEXG3P6+qtaq6D8gFZohIDDAXeBJAVetUtcyH1+AV45OjSIwKpaiylj1Fx/1dHWOM6RVfBsdI4JDH5zx3W7vHqGoDUA4knObccUAR8JSIfCwiT4hIZHtfLiJLRCRLRLKKioq8cT09JiLMGueO57B2DmPMAOfL4JB2trV9TtPRMR1tDwLOAx5V1WnAceCUthMAVX1cVTNVNTMpKanrtfYRW07WGDNY+DI48oDRHp9HAfkdHSMiQUAsUHqac/OAPFXd4G5fiRMk/V5zO8d6a+cwxgxwvgyOTcAEEUkXkRCcxu7VbY5ZDSx2318PvKvOb9XVwCK311U6MAHYqKpHgUMiMsk9Zx6ww4fX4DXpiZEkR4dScryOzwqq/F0dY4zpMZ8Fh9tmcS+wBqfn04uqul1ElovIVe5hTwIJIpILfAv3sZOqbgdexAmFN4F7VLXRPefrwLMi8imQAfzcV9fgTSLi0S232M+1McaYnpOh8NgkMzNTs7Ky/F0Nnt94kKUvb2X+lOE8dut0f1fHGGM6JCKbVTWzvX02crwPtTSQ7yuhqWnwB7YxZnCy4OhITTm8cAt88oLXihwzLILU2DDKquvZdbTSa+UaY0xfsuDoSEg0lOfB2z+COu8M2hMRZtlyssaYAc6CoyMBATD/QajMhw8f8lqxs205WWPMAGfBcTpjZsHZ18G/fwdlB71SZHM7x4Z9JTRaO4cxZgCy4OjMF38MCLz1Q68UNyo+gtHDwqmsaWBHfoVXyjTGmL5kwdGZuNEw5xuw/WU48JFXipyV3tzOYeM5jDEDjwVHV8y5D2JGwptLoamp18XZ+hzGmIHMgqMrQiKdR1ZHsuGTv/W6uObg2LT/GA2NvQ8iY4zpSxYcXXXO9TBqBrz9Y6jpXdvEiNhw0hIiqKptYJu1cxhjBpguBYeI3CciMeJ4UkS2iMilvq5cvyICCx6E44Xwwa97XZw9rjLGDFRdveO4Q1UrgEuBJOCrwIM+q1V/NXI6TL0Z1v8BSvf2qqhZ42wgoDFmYOpqcDQvrHQ58JSqfkL7iy0NfvN+CAHBsPb7vSqmeSBg1v5S6q2dwxgzgHQ1ODaLyFqc4FgjItHA0PxtFzMCLvwm7PoH7FvX42KSY8I4IymS6rpGPs3r98umG2NMi64Gx504a2Wcr6rVQDDO46qhafa9EDsG3lwGTY2dH99RMdbOYYwZgLoaHLOB3apaJiK3AP8FlPuuWv1ccDhc+hMo2AZbnu5xMbPHJQLWzmGMGVi6GhyPAtUiMhX4LnAA+IvPajUQTF4IY+fAuz+FEz171DRz3DAAsvYfo7ah53cuxhjTl7oaHA3uWuALgYdV9WEg2nfVGgBEYP4voLoU1v1Pj4pIjAplYkoUtQ1NZB+0dg5jzMDQ1eCoFJFlwK3AayISiNPOMbSNmArn3QobHoPinB4VMdu65RpjBpiuBsdNQC3OeI6jwEigZ//MHmy+8H0ICoc13+vR6S3LyVpwGGMGiC4FhxsWzwKxInIlUKOqQ7uNo1lUMlz0HchZA7lvd/v0mekJiMCWg2XU1Fs7hzGm/+vqlCM3AhuBG4AbgQ0icn0XzpsvIrtFJFdElrazP1REXnD3bxCRNI99y9ztu0XkMo/t+0Vkq4hki0hWV+rvczPvhvh0ePM/obG+W6fGR4Zw5vAY6hqa2HLwmI8qaIwx3tPVR1XfwxnDsVhVbwNmAKcdOu22gzwCLAAmAzeLyOQ2h90JHFPV8cBvgV+6504GFgFTgPnAH9zyml2sqhmqmtnF+vtWUChc9jMo3g1ZK7p9enM7x3obz2GMGQC6GhwBqlro8bmkC+fOAHJVda+q1gHP4/TK8rQQaB4IsRKYJyLibn9eVWtVdR+Q65bXf026HNIvgvd+7vS06oaWgYDWzmGMGQC6GhxvisgaEbldRG4HXgNe7+SckcAhj8957rZ2j1HVBpxBhQmdnKvAWhHZLCJLOvpyEVkiIlkiklVUVNRJVb1ABOY/CLUV8P4vunXqjPRhBAhkHyrjRJ21cxhj+reuNo5/B3gcOBeYCjyuqg90clp7kyBqF4853blzVPU8nEdg94jI3A7q/LiqZqpqZlJSUidV9ZKUyZB5B2x6Egp3dvm02PBgpqTGUt+obNrfvbsVY4zpa11eyElVX1LVb6nqN1V1VRdOyQNGe3weBeR3dIyIBAGxQOnpzlXV5tdCYBX97RHW5/8TQqOceay0bU527IIJzvQjP/7f7Rw7Xuer2hljTK+dNjhEpFJEKtr5qRSRzpau2wRMEJF0EQnBaexe3eaY1cBi9/31wLvuCPXVwCK311U6MAHYKCKR7sy8iEgkzvog27pzwT4XmQCfXwZ734PP3uzyaXdfdAaTUqLZU3ScO5/eZI+sjDH91mmDQ1WjVTWmnZ9oVY3p5NwG4F5gDbATeFFVt4vIchG5yj3sSSBBRHKBb+HMwIuqbgdeBHYAbwL3qGojkAJ8KCKf4HQPfk1Vu/7bua+cfxckTnQGBTZ07e4hNjyYp++YQWpsGFsOlvH15z629ciNMf2SaDcepwxUmZmZmpXVx0M+ct6CZ6+HS38Kn/t6l0/LLazkukc/ovxEPYvOH80vrj0Hp6OZMcb0HRHZ3NGQhy63cZhumnAJjL8E/vnfUNX1Xl3jk6NZcXsmoUEBPL/pEL99u2dzYBljjK9YcPjSZT+H+mp476fdOm362GH8/svnESDwu3dyeGb9AR9V0Bhjus+Cw5eSJsKMJbDlL3B0a7dOvWRyCj+/5hwAfvDqNt7cdtQXNTTGmG6z4PC1i74LYXHd7p4LsGjGGL51yUSaFL7x/Mc2xsMY0y9YcPhaeDx84Xuw/wPY2bY3cue+/oXxfGXmGOoamrjzz5vYfbTSB5U0xpius+DoC+fdDsmTYe1/QX1Nt04VEZYvPJtLJ6dQUdPA4hUbyS874Zt6GmNMF1hw9IXAIGeZ2bKDsP6R7p8eIPzu5mmcnxbP0YoabluxkbJqG11ujPEPC46+Mu7zMOkKWPdrqOx+Q3dYcCBP3HY+E1OiyC2s4q6ns2zhJ2OMX1hw9KVLfwKNdfDO8h6dHhsRzJ+/OoMRsWFkHThmo8uNMX5hwdGXEs6AWf8B2c/C4S09KiI1Lpy/3DGD2PBg3tpRwPdf3c5QGP1vjOk/LDj62tzvQGRSj7rnNpuQEs2Ti53R5c9tPMjD79jocmNM37Hg6GthMTDvB3BoPWx7qcfFZKYN43c3TyNA4KG3c/jbhoNerKQxxnTMgsMfMr4Cw8+Ft34AddU9LuayKcP5ydVnA/Bfr2xl7XYbXW6M8T0LDn8ICIQFv4SKw/Dv3/WqqK/MHMt98ybQpPD15z4my0aXG2N8zILDX8Z+DiZfDR8+BOV5vSrq/i9O4OYZo6ltaOLOp7PIKbDR5cYY37Hg8KdLloM2wds/6lUxIsJPFp7NF89KofxEPYtXbORIuY0uN8b4hgWHP8WPdRZ52vp3OLSxV0UFBQbw+y9PY/rYePLLa1i8YiPl1fVeqqgxxrSy4PC3C74JUcPhjQegqXeD+cKCA3lycSbjk6P4rKCKr/3FRpcbY7zPgsPfQqPgiz+C/C3w6Qu9Li4uIoSn75jB8JgwNu4v5b7nP6axyQYIGmO8x4KjPzj3Jhg53WnrqCzodXEj48J5+o4ZRIcFsWZ7AT94dZuNLjfGeI0FR38QEACX/w/UlMFjcyD3nV4XOWl4NE/clklIUADPbjjI79/N9UJFjTHGx8EhIvNFZLeI5IrI0nb2h4rIC+7+DSKS5rFvmbt9t4hc1ua8QBH5WET+4cv696mR0+Fr70FEIjxzLbz1Q2jsXeP2zHEJ/G5RBiLw67c+4/mNNrrcGNN7PgsOEQkEHgEWAJOBm0VkcpvD7gSOqep44LfAL91zJwOLgCnAfOAPbnnN7gN2+qrufpMyGb72Lky/Hf71EDy1AI7t71WR888ewfKFzujy/1y1lbd39P5RmDFmaPPlHccMIFdV96pqHfA8sLDNMQuBp933K4F5IiLu9udVtVZV9wG5bnmIyCjgCuAJH9bdf0Ii4EsPw/VPQdFueGwubH+lV0XeOmssX//CeJoU7vnbFjYfsNHlxpie82VwjAQOeXzOc7e1e4yqNgDlQEIn5z4EfBcY3AtRnH0t3P0BJI6Hvy+G/70f6ns+qO9bl0zkpszW0eW5hTa63BjTM74MDmlnW9uuPR0d0+52EbkSKFTVzZ1+ucgSEckSkayioqLOa9sfxafBV9+Ez30DNj8Ff/oCFO7qUVEiws+uOZt5ZyZTVl3P4hWbOFrevfXPjTEGfBscecBoj8+jgPyOjhGRICAWKD3NuXOAq0RkP86jry+IyDPtfbmqPq6qmaqamZSU1Pur8ZegEGflwFtegqpCePzzsPnpHq3l4YwuP49pY+I4XHaCW57cwMcHj3m/zsaYQc2XwbEJmCAi6SISgtPYvbrNMauBxe7764F31RlwsBpY5Pa6SgcmABtVdZmqjlLVNLe8d1X1Fh9eQ/8x/ovwH/+C0TPgf78BK++AmvJuFxMeEsiKxeczPtlZu/yaP/ybb76QbXNbGWO6zGfB4bZZ3AuswekB9aKqbheR5SJylXvYk0CCiOQC3wKWuuduB14EdgBvAveoqs2dET0cbl3lLAS141X441w43OlTu1PER4bwyj1z+L+fP4OQoABWfXyYi3/1Pg+9/Rkn6uyP2RhzejIURhRnZmZqVlaWv6vhXQc3wEt3QuURZ8qSWfc4Awm76VBpNb94Yyevb3UWgRoRG8YD88/kqqmpBAS019RkjBkKRGSzqma2u8+CYwA7cQxevRd2/QPGXwJXPwpRPWvP2bC3hJ+8toNthysAyBgdxw++NJnzxsR7s8bGmAHCgmOwBgc4jeSbnoA134PweLj2cRh3UY+KampSVm7J43/W7KaoshaAhRmpPDD/TFLjwr1Za2NMP2fBMZiDo9nRrfD3r0JJLsz9Nly0FAKDelRUVW0Dj76fy58+2EddQxNhwQH8n7ln8H8uGkdESM/KNMYMLBYcQyE4AOqOw+vfhexnYPQsuO4JiBvd+XkdOFRazYNv7uK1T48AMDwmjAcWTGLh1JHW/mHMIGfBMVSCo9mnL8I/vgkBQbDwETjryl4Vt3FfKT/5xw62Hna6/04dHccPrpzM9LHW/mHMYGXBMdSCA6BkjzPW40g2zFgCl/wEgsN6XFxTk/LSljz+26P946qpqSxdYO0fxgxGFhxDMTgAGuqcxaHWPwIp58ANT0HihF4Veby2gUff38PjH+xtaf9YMvcM7rb2D2MGFQuOoRoczT5bA6vuhoZauOJXMPVmkN61UeQdq+bBN3bxD7f9IyUmlAfmn8nVGdb+YcxgYMEx1IMDoCIfXvoaHPjQWar2il9DaHSvi92032n/+DTPbf8YFcsPvjSZ6WOH9bpsY4z/WHBYcDiaGmHdr+CfDzoz7177BIya3vtim5SXPz7Mf7+5i0K3/eNLbvvHSGv/MGZAsuCw4DjZ/n/BS3dBZT6kXQgzvgaTrujxuI9mx2sbeOyfe3h83V5qG5oIDQpgydxx3H3RGUSGWvuHMQOJBYcFx6lOHHOmZ9/0JJQfhJiRkPlVOO/2Hk9b0izvWDW/fHM3//uJM4t+Skwo373sTK6ZZu0fxqD7V48AABXfSURBVAwUFhwWHB1ranQazzc+Dnvfg8AQmHKt04W3l4+xstz2j0/c9o+xCREsnJrKVRkjGZ8c5Y3aG2N8xILDgqNrij5z5r3K/hvUVULqeU6ATLmmx2NAmpqUV7IP86s1u8n3WHFwSmoMV2eM5MqpIxgRa+0gxvQ3FhwWHN1TWwmfPA8b/wTFuyEiAc5bDJl39HgKk8YmZf3eEl7NPswb245SWdMAOL2CZ6YPY2HGSBacPZy4iBBvXokxpocsOCw4ekYV9v3TCZDdrzvbJl3u3IWkz+3xWJCa+kbe313E6k8O8/bOQuoamgAIDhQumpjMwoxUvnhWCuEhgd66EmNMN1lwWHD0XtlByFrhNKifKIXESU5vrKmLejUepKKmnrXbC3g1+zD/yi2myf3rGBkSyKVThrMwI5U54xMJDvTlKsfGmLYsOCw4vKe+Bra/DBv+6MyDFRINGV92QqSX05kUVtbw2qdHeDU7n+xDZS3bEyJDuOLcESzMSOW8MfFIL0e9G2M6Z8FhweF9qs565xsfh20vQ1M9jLvYeYw18TII6N1jpgMlx1mdnc8r2YfZU3S8Zfuo+HCumprKwoyRTBre+5Hvxpj2WXBYcPhWVSFseRo2rXAGFcaOgfPvhPNug4jeTT2iqmzPr2D1J/mszs7naEVrz6wzh0ezMGMkX5o6glHxEb29CmOMBwsOC46+0dgAu19zGtP3fwBBYXDO9XD+1yA1o9fFNzUpG/eX8mp2Pq9vPUL5ifqWfeenxXNVxkiuOGcEwyKtZ5YxvWXBYcHR9wp2wKY/Od1666th1Aw45wY442JIGN/r2XnrGppY91kRr2Qf5u2dBdTUOz2zggKECyckclVGKhdPSrbuvcb0kN+CQ0TmAw8DgcATqvpgm/2hwF+A6UAJcJOq7nf3LQPuBBqBb6jqGhEJA9YBoUAQsFJVf9hZPSw4/OhEGXzynDO1SUmOsy1mFIz7vBMi6Rf1eoqTqtoG3tpxlFez8/kgp5hGt2uWCJwzMpYLJyRywfgkpo+NJyTIemcZ0xV+CQ4RCQQ+Ay4B8oBNwM2qusPjmP8LnKuqd4vIIuAaVb1JRCYDzwEzgFTgbWAi0AREqmqViAQDHwL3qer609XFgqOfKN3nTGuy5z3Ytw5q3J5TKefAGZ93wmTM5yCk5+0VJVW1vL71CK9vPcrmA8eoa2xq2RceHMisccO4YEISF05IZEJylPXQMqYD/gqO2cCPVPUy9/MyAFX9hccxa9xjPhKRIOAokAQs9TzW8ziPcyNwguM/VHXD6epiwdEPNTU63Xn3vAd734dDG6Cxzpkra8wsJ0TGXQwjpva4h1Z1XQMb95XyQU4xH+YUs7ug8qT9KTGhXDA+ibkTE5kzPpHEqNBeX5Yxg4W/guN6YL6q3uV+vhWYqar3ehyzzT0mz/28B5gJ/AhYr6rPuNufBN5Q1ZXuncxmYDzwiKo+0MH3LwGWAIwZM2b6gQMHfHKdxkvqjsOBj5w7kr3vQ8E2Z3t4vDNKfdzFTpgMS+/xVxRU1PBhTjEf5hbzQU4xxVW1J+0/a0QMcyckcsGERM5PG0ZYsI1cN0PX6YLDl4sktPcMoG1KdXRMh+eqaiOQISJxwCoROVtVt51ysOrjwOPg3HF0p+LGD0IiYcIXnR9wuvju/Wfro60drzrb49NaQyR9bre6+6bEhHHd9FFcN30Uqsquo5V8mFPMupwiNu4rZeeRCnYeqeCP6/YSGhTAjPRhXDA+kQsnJHHm8GibEt4Yly+DIw/wnBFvFJDfwTF57qOqWKC0K+eqapmIvA/MB04JDjPARSXDuTc4P6pQnNN6N7J1JWx+ChBIndba0D56JgR17XGTiHDWiBjOGhHD1+aOo6a+kc0HjrEup4gPc4rZnl/BBznOnckv3thFYlQIc9wQuXBCIikxPZst2JjBwJePqoJwGsfnAYdxGse/rKrbPY65BzjHo3H8WlW9UUSmAH+jtXH8HWACMAyod0MjHFgL/FJV/3G6ulgbxyDTWO+MWt/7vnM3krcJtBGCwmHs55wQGfd5SJ4CAT3rRVVcVcu/cotbHm0d8ZgSHmBCclRLiMwcN4yIEFvh0Awu/uyOeznwEE533BWq+jMRWQ5kqepqt3vtX4FpOHcai1R1r3vu94A7gAbgflV9Q0TOBZ52ywsAXlTV5Z3Vw4JjkKupgAP/chva34Piz5zt4cMgbY6zPG7aBZB0Vo+CRFXZU1TV0sj+0d4SqusaW/YHBwoZo+OYkhrLlNQYJqfGMCE52rr+mgHNBgBacAwt5Yed6eD3f+iMYC876GyPSICxHkGSfFaPBiLWNTTx8cFjfJhbzLqcYrbmlbXM6tssJDCACSlRTEmNaQmUs0bE2NrrZsCw4LDgGNqOHXBDxA2S8kPO9ojENnckZ/YoSMqq68g+VMb2/Ap2HKlgR34F+4qPn3KcCKQnRHJWasxJgWLdgE1/ZMFhwWE8HdvfGiT7PoCKPGd7RKITIGkXOGGSNKnHU6NU1Taw80gF2w+Xsz2/gu35FeQUVlLfeOr/bykxoS0hMiU1hskjYhk9LNwGJxq/suCw4DAdUT05SPZ/ABWHnX2RSR5BMtdZb6QXv8zrGprIKax07kzyK9ieX86O/AqOe7SXNIsOC2LyiNa7kikjYzgjKcoWtDJ9xoLDgsN0lSoc23fyHUml2xM8Mrk1SNLnemWyxqYm5UBpNdvzW+9MduSXU1xVd8qxIUEBnDk8mskjYjhzeDQTU6KZkBJNYlSI3Z0Yr7PgsOAwPaUKpXtPviOpPOLsi0ppDZLRs5w7ksBgr3xtYUWNGyStgXKwtLrdY+MigpmYHM34lCgmJkcxMcV5nxQVaoFiesyCw4LDeEtLkHzQekdSddTZFxDs3IUknwnJk53G9uTJzjQpvVwREZz12Xe4j7k+K6jks4JKcgqrqKxpaPd4CxTTGxYcFhzGV1ShZA/kb4HCHVC4C4p2Ou0mzQJDIXGi0/3XM1TixvZ4gGLr1ysFFbXkFFbyWUEVOW6YfFZQedpAmZAcxYSUaCa6rxMsUEwbFhwWHKav1R2Hot1QuNMJksJdzvvmHlwAwRFOz62ks9xQOcsJlNhRvW47sUAxvWXBYcFh+ouaCjdQdkDRrta7lObHXQChMU6gJJ/lhop7lxKV4pdAiQkLIjUunOSYMFKiQ0mOCSUlJozk6LCW90lRoTZSfpCx4LDgMP1ddakbJDtbXwt3QnVx6zFhcU6AJJ/ptKXEpzvtJ3Fje7X4FfQsUNoaFhlCcnRzqDivKTGhJEU7rykxYSRFh1qX4gHCgsOCwwxUVUUej7p2tIZK8+qJzaKGOyESn+5MPe/5PjKxx3cqqkpxVR0FFTUUVtZQUFFLYUUtBZU1FFbUUFhZS0FFDUWVtadMu9KRhMgQ5+4lJrQ1aNqGTVQoQRYwfmXBYcFhBhNVOHHMWYr3mPtTut9pkD+2r3UAY7OQKDdExp4aLrGjvdKFuLFJKTnuhopHoBRU1FLkBk5BRQ3FVV0LmACB5OgwhseGMTzGeR0R2/p5RGw4yTGhttiWD1lwWHCYoaS+xpnY8dg+N1z2n/y+0WPlQwmEuNFOkDQ/+vJ8Hxrt1ao1NiklVbXOnYtHoBRW1lJYUUNBZQ1Hy2spOV5LV341DYsMcYOkNVxS3GAZ7m6Lsokle8SCw4LDGEdTk9MQ33K3sv/k99UlJx8fkeAESdRwZ3GtqBSISnJeI5PdbcnOCo5eVNfQRGFlDUfLazha4bweKT/5c0FFDQ1duH2JDg1qCZERLXcw4S0hkxoXRmx4sPUca8OCw4LDmK6pKXfvUPZ7BMoBZynf44VwvJhTV4DGeRwWlewRJimtoRLpETiRyRDsndUTm+9ejpwSLidO+lzb0NRpWeHBgYyICyPVDZQRceGktnkdancuFhwWHMZ4R2OD09OrqtD9KXACxfNzc8icONZ+GWGxJ4dJc8g0b4tIgLAYp1tyWAwEhfWqcb+sur7dYDnicRdTVdt5z7HosCBSY8NJjfMIlNjwlsAZHhs2qNpcLDgsOIzpew11cLzo5DCpKnB6ip20rRBqKzouJyD45CAJjXHCp+VzdJt9MRAae/Ln4IjThk9FTT1HymrILz/BkbIajpSfIN99PVJeQ37ZiS7duSREhjAizgmU5juVEbFhpMaFkxoXTkr0wOktdrrgGFr3XsaYvhMUArEjnZ/O1J9ovWupLnGCpKbcfa049bV038nb2nt85kkCTw2f5vfRI4iJH0tM3FgmJabBGaOduntQVUqP17WEyJHyU0OmoKKGkuN1lByvY9vh9oNQBIZFhJAQFUJCZCiJ0aEkRIaQGBVCYlQoCVGhJESFkOS+9te17PtnrYwxQ0twuNNdOH5s989taoK6qg5Cpr3wqXTel+dBQbkz23FTfWt5EgDRqW7vsrEQNxaJTyMhfiwJ8Wmcndr+CP7GJqW4qrY1WNxXz7uXwsralnCBqk4vLTw4kMRoN2RawqU1dBIjQ0iIcvbFRYQQGNA3DfwWHMaYgS0gwLlzCIuB2B6c39TohMex/U5HgLIDre/3vNs6jX6zoDCIG+MES5wbdvFpBMaNJSV+LClj4pnWwVc1NDZRWl1HSZXzU1xVS3GVEybFzaFSVUtxVR1FVbWcqG/kUOkJDpWe6PyPQWCYGzAJzSETGcpdF6aTGhfegz+YjllwGGOGtoBAZ2LJ2FHO2ipt1Z+AskMegbK/9f3BDVBbfvLx4fEnBUrr+3SCYkc7c3xFd96zTFU5XtfoBkotRZV1lByvbQmctsFTVl3f8tnTV2aN6emfTId8GhwiMh94GAgEnlDVB9vsDwX+AkwHSoCbVHW/u28ZcCfQCHxDVdeIyGj3+OFAE/C4qj7sy2swxgxxweGQNNH5ac+JY+3frRRsh91vQGOb1RzDYp2eYxEJzjr3EQkQMcxjm/MjEQlERQwjalgcaYmdj5Opb2yi9PjJoVJSVcfwGO90f/bks+AQkUDgEeASIA/YJCKrVXWHx2F3AsdUdbyILAJ+CdwkIpOBRcAUIBV4W0QmAg3A/1PVLSISDWwWkbfalGmMMX0nPN75SW3nAVVTk/OoqzlQyg45jf/Vxc5rRR4c/dQZH9NYe+r54DTsnxQsp4YMEQkERySQEpFASkICjOj5/GRd4cs7jhlArqruBRCR54GFgOcv+YXAj9z3K4HfizN8cyHwvKrWAvtEJBeYoaofAUcAVLVSRHYCI9uUaYwx/UNAQGvPsrGf6/g4VaivdkOlBI6XtL4/6acUinOheoPzWRvbLy8w1B31PxbueNPrl+XL4BgJHPL4nAfM7OgYVW0QkXIgwd2+vs25J/XpE5E0YBqwwZuVNsaYPifiTNsSEuk0vHdFU5PTvlJd2kHIlDg9xHzAl8HR3n1S287WHR1z2nNFJAp4CbhfVdvtMC0iS4AlAGPGeL9xyBhj/CogoPUxWcIZffvVPiw7Dxjt8XkUkN/RMSIShNOZrvR054pIME5oPKuqL3f05ar6uKpmqmpmUlJSLy/FGGNMM18GxyZggoiki0gITmP36jbHrAYWu++vB95VZw6U1cAiEQkVkXRgArDRbf94Etipqr/xYd2NMcZ0wGePqtw2i3uBNTjdcVeo6nYRWQ5kqepqnBD4q9v4XYoTLrjHvYjT6N0A3KOqjSJyAXArsFVEst2v+k9Vfd1X12GMMeZkNsmhMcaYU5xuksOBMU2jMcaYfsOCwxhjTLdYcBhjjOkWCw5jjDHdMiQax0WkCDjg73p0QSJQ7O9K+NBgvj67toFrMF9fb65trKq2OwhuSATHQCEiWR31YhgMBvP12bUNXIP5+nx1bfaoyhhjTLdYcBhjjOkWC47+5XF/V8DHBvP12bUNXIP5+nxybdbGYYwxplvsjsMYY0y3WHAYY4zpFguOfkBERovIeyKyU0S2i8h9/q6Tt4lIoIh8LCL/8HddvE1E4kRkpYjscv8bzvZ3nbxFRL7p/p3cJiLPiUiYv+vUGyKyQkQKRWSbx7ZhIvKWiOS4r/H+rGNPdXBt/+P+vfxURFaJSJw3vsuCo39oAP6fqp4FzALuEZHJfq6Tt90H7PR3JXzkYeBNVT0TmMoguU4RGQl8A8hU1bNxlkdY5N9a9dqfgfltti0F3lHVCcA77ueB6M+cem1vAWer6rnAZ8Ayb3yRBUc/oKpHVHWL+74S5xfPyNOfNXCIyCjgCuAJf9fF20QkBpiLs7YMqlqnqmX+rZVXBQHh7gqdEZy6iueAoqrrcNb+8bQQeNp9/zRwdZ9WykvauzZVXauqDe7H9TirqfaaBUc/IyJpwDRgg39r4lUPAd8FmvxdER8YBxQBT7mP4p4QkUh/V8obVPUw8CvgIHAEKFfVtf6tlU+kqOoRcP4RByT7uT6+cgfwhjcKsuDoR0QkCmc99ftVtcLf9fEGEbkSKFTVzf6ui48EAecBj6rqNOA4A/dRx0ncZ/0LgXQgFYgUkVv8WyvTEyLyPZxH4s96ozwLjn5CRIJxQuNZVX3Z3/XxojnAVSKyH3ge+IKIPOPfKnlVHpCnqs13iCtxgmQw+CKwT1WLVLUeeBn4nJ/r5AsFIjICwH0t9HN9vEpEFgNXAl9RLw3cs+DoB0REcJ6R71TV3/i7Pt6kqstUdZSqpuE0rL6rqoPmX62qehQ4JCKT3E3zgB1+rJI3HQRmiUiE+3d0HoOk4b+N1cBi9/1i4FU/1sWrRGQ+8ABwlapWe6tcC47+YQ5wK86/xrPdn8v9XSnTZV8HnhWRT4EM4Od+ro9XuHdRK4EtwFac3xcDenoOEXkO+AiYJCJ5InIn8CBwiYjkAJe4nwecDq7t90A08Jb7e+Uxr3yXTTlijDGmO+yOwxhjTLdYcBhjjOkWCw5jjDHdYsFhjDGmWyw4jDHGdIsFhzH9mIh8fjDOKGwGNgsOY4wx3WLBYYwXiMgtIrLRHWT1R3f9kSoR+bWIbBGRd0QkyT02Q0TWe6yREO9uHy8ib4vIJ+45Z7jFR3ms9/GsO4rbGL+x4DCml0TkLOAmYI6qZgCNwFeASGCLqp4H/BP4oXvKX4AH3DUStnpsfxZ4RFWn4swJdcTdPg24H5iMMxvvHJ9flDGnEeTvChgzCMwDpgOb3JuBcJyJ8pqAF9xjngFeFpFYIE5V/+lufxr4u4hEAyNVdRWAqtYAuOVtVNU893M2kAZ86PvLMqZ9FhzG9J4AT6vqSauricj32xx3uvl9Tvf4qdbjfSP2/63xM3tUZUzvvQNcLyLJ0LKG9Vic/7+ud4/5MvChqpYDx0TkQnf7rcA/3fVX8kTkareMUBGJ6NOrMKaL7F8uxvSSqu4Qkf8C1opIAFAP3IOzqNMUEdkMlOO0g4AzdfdjbjDsBb7qbr8V+KOILHfLuKEPL8OYLrPZcY3xERGpUtUof9fDGG+zR1XGGGO6xe44jDHGdIvdcRhjjOkWCw5jjDHdYsFhjDGmWyw4jDHGdIsFhzHGmG75/8/BFkl4bl6lAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "from torch.nn import init\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "mnist_train = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=True, download=True, transform=transforms.ToTensor())\n",
    "mnist_test = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=False, download=True, transform=transforms.ToTensor())\n",
    "\n",
    "batch_size = 200\n",
    "train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False,num_workers=0)\n",
    "\n",
    "num_inputs, num_outputs, num_hiddens1, num_hiddens2 = 784, 10, 256, 256\n",
    "drop_prob1, drop_prob2 = 0.3, 0.5\n",
    "\n",
    "\n",
    "class FlattenLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(FlattenLayer, self).__init__()\n",
    "    def forward(self, x):\n",
    "        return x.view(x.shape[0], -1)\n",
    "\n",
    "\n",
    "net = nn.Sequential(\n",
    "     FlattenLayer(),\n",
    "     nn.Linear(num_inputs, num_hiddens1),\n",
    "     nn.ReLU(),\n",
    "     nn.Dropout(drop_prob1),\n",
    "     nn.Linear(num_hiddens1, num_hiddens2),\n",
    "     nn.ReLU(),\n",
    "     nn.Dropout(drop_prob2),\n",
    "     nn.Linear(num_hiddens2, num_outputs),\n",
    "     )\n",
    "\n",
    "for params in net.parameters():\n",
    "    init.normal_(params, mean=0, std=0.1)\n",
    "\n",
    "\n",
    "def evaluate_accuracy(data_iter, net):\n",
    "    acc_sum, n = 0.0, 0\n",
    "    for X, y in data_iter:\n",
    "        if isinstance(net, torch.nn.Module):\n",
    "            net.eval()\n",
    "            acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()\n",
    "            net.train()\n",
    "        else:\n",
    "            if('is_training' in net.__code__.co_varnames):\n",
    "                acc_sum += (net(X, is_training=False).argmax(dim=1) == y).float().sum().item()\n",
    "            else:\n",
    "                acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()\n",
    "        n += y.shape[0]\n",
    "    return acc_sum / n\n",
    "\n",
    "loss = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(net.parameters(), lr=0.01)\n",
    "num_epochs = 12\n",
    "\n",
    "def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, optimizer):\n",
    "    train_ls, test_ls, x_epoch = [], [], []\n",
    "    for epoch in range(num_epochs):\n",
    "        train_1_sum, train_acc_sum, n = 0.0, 0.0, 0\n",
    "        train_1_test_sum, n_test = 0.0, 0\n",
    "        for X, y in train_iter:\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat, y).sum()\n",
    "            optimizer.zero_grad()\n",
    "            l.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            train_1_sum += l.item()\n",
    "            train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()\n",
    "            n += y.shape[0]\n",
    "        train_ls.append(train_1_sum / n)\n",
    "        x_epoch.append(epoch + 1)\n",
    "        test_acc = evaluate_accuracy(test_iter, net)\n",
    "        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f' % (epoch + 1, train_1_sum / n, train_acc_sum / n, test_acc))\n",
    "\n",
    "        for X_test, y_test in test_iter:\n",
    "            y_hat = net(X_test)\n",
    "            l = loss(y_hat, y_test).sum()\n",
    "            train_1_test_sum += l.item()\n",
    "            n_test += y_test.shape[0]\n",
    "        test_ls.append(train_1_test_sum / n_test)\n",
    "\n",
    "\n",
    "    plt.plot(x_epoch, train_ls, label=\"train_loss\", linewidth=2)\n",
    "    plt.plot(x_epoch, test_ls, label=\"test_loss\", linewidth=1.5)\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.ylabel(\"loss\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}